# %% [markdown]
# # Chipotle 数据分析报告
#
# 本笔记本对 `data/chipotle.tsv` 文件进行详细的数据分析
#
# ## 分析内容：
# 1. 数据加载与基本信息
# 2. 数据清洗与预处理
# 3. 订单分析
# 4. 商品分析
# 5. 价格分析
# 6. 客户行为分析
# 7. RFM 分析
# 8. 时间序列分析（如果有时间数据）
# 9. 关联规则分析
# 10. 数据可视化

# %% [markdown]
# ## 1. 数据加载与基本信息

# %%
# 导入必要的库（仅使用 Python 标准库和基础功能）
import csv
from collections import Counter, defaultdict
from datetime import datetime
import statistics
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import math


def load_data(file_path='data/chipotle.tsv'):
    """加载数据"""
    print("=" * 80)
    print("1. 数据加载")
    print("=" * 80)

    df = pd.read_csv(file_path, sep='\t')
    print(f"✓ 成功加载数据文件: {file_path}")
    print(f"✓ 数据形状: {df.shape[0]} 行 × {df.shape[1]} 列\n")

    return df


def basic_info(df):
    """基本信息分析"""
    print("=" * 80)
    print("2. 数据基本信息")
    print("=" * 80)

    print("\n【数据前5行】")
    print(df.head())

    print("\n【数据类型】")
    print(df.dtypes)

    print("\n【数据统计信息】")
    print(df.describe())

    print("\n【缺失值统计】")
    missing = df.isnull().sum()
    print(missing)
    print(f"\n总缺失值: {missing.sum()}")

    print("\n【列名】")
    print(df.columns.tolist())
    print()


def clean_data(df):
    """数据清洗"""
    print("=" * 80)
    print("3. 数据清洗")
    print("=" * 80)

    # 清洗价格列，去除 $ 符号并转换为浮点数
    df['item_price_clean'] = df['item_price'].str.replace('$', '').str.strip().astype(float)
    print("✓ 清洗价格数据，去除 $ 符号并转换为数值类型")

    # 计算每行的总价
    df['total_price'] = df['quantity'] * df['item_price_clean']
    print("✓ 计算每行总价 (数量 × 单价)")

    # 处理 NULL 值
    df['choice_description'] = df['choice_description'].replace('NULL', np.nan)
    print("✓ 将 'NULL' 字符串替换为 NaN\n")

    return df


def order_analysis(df):
    """订单分析"""
    print("=" * 80)
    print("4. 订单分析")
    print("=" * 80)

    # 订单数量
    total_orders = df['order_id'].nunique()
    print(f"✓ 总订单数: {total_orders:,}")

    # 总商品数
    total_items = df.shape[0]
    print(f"✓ 总商品条目数: {total_items:,}")

    # 平均每单商品数
    avg_items_per_order = total_items / total_orders
    print(f"✓ 平均每单商品数: {avg_items_per_order:.2f}")

    # 每单商品数分布
    items_per_order = df.groupby('order_id').size()
    print(f"\n【每单商品数统计】")
    print(f"  最少: {items_per_order.min()} 件")
    print(f"  最多: {items_per_order.max()} 件")
    print(f"  中位数: {items_per_order.median():.0f} 件")
    print(f"  平均: {items_per_order.mean():.2f} 件")

    # 订单金额分析
    order_totals = df.groupby('order_id')['total_price'].sum()
    print(f"\n【订单金额统计】")
    print(f"  最低订单金额: ${order_totals.min():.2f}")
    print(f"  最高订单金额: ${order_totals.max():.2f}")
    print(f"  平均订单金额: ${order_totals.mean():.2f}")
    print(f"  中位数订单金额: ${order_totals.median():.2f}")
    print(f"  总营业额: ${order_totals.sum():,.2f}")
    print()

    return items_per_order, order_totals


def item_analysis(df):
    """商品分析"""
    print("=" * 80)
    print("5. 商品分析")
    print("=" * 80)

    # 商品种类
    unique_items = df['item_name'].nunique()
    print(f"✓ 商品种类数: {unique_items}")

    # 最受欢迎的商品（按订单数）
    item_counts = df['item_name'].value_counts()
    print(f"\n【最受欢迎的前10种商品】（按订单次数）")
    for i, (item, count) in enumerate(item_counts.head(10).items(), 1):
        print(f"  {i:2d}. {item:40s} - {count:4d} 次")

    # 按销售数量统计
    item_quantity = df.groupby('item_name')['quantity'].sum().sort_values(ascending=False)
    print(f"\n【销售数量最多的前10种商品】")
    for i, (item, qty) in enumerate(item_quantity.head(10).items(), 1):
        print(f"  {i:2d}. {item:40s} - {qty:4d} 件")


    # 按销售额统计
    item_revenue = df.groupby('item_name')['total_price'].sum().sort_values(ascending=False)
    print(f"\n【销售额最高的前10种商品】")
    for i, (item, revenue) in enumerate(item_revenue.head(10).items(), 1):
        print(f"  {i:2d}. {item:40s} - ${revenue:8.2f}")

    # 平均价格
    item_avg_price = df.groupby('item_name')['item_price_clean'].mean().sort_values(ascending=False)
    print(f"\n【平均价格最高的前10种商品】")
    for i, (item, price) in enumerate(item_avg_price.head(10).items(), 1):
        print(f"  {i:2d}. {item:40s} - ${price:6.2f}")

    print()
    return item_counts, item_quantity, item_revenue, item_avg_price


def price_analysis(df):
    """价格分析"""
    print("=" * 80)
    print("6. 价格分析")
    print("=" * 80)

    print(f"【价格统计】")
    print(f"  最低价格: ${df['item_price_clean'].min():.2f}")
    print(f"  最高价格: ${df['item_price_clean'].max():.2f}")
    print(f"  平均价格: ${df['item_price_clean'].mean():.2f}")
    print(f"  中位数价格: ${df['item_price_clean'].median():.2f}")

    # 价格区间分布
    price_bins = [0, 5, 10, 15, 20, 100]
    price_labels = ['$0-5', '$5-10', '$10-15', '$15-20', '$20+']
    df['price_range'] = pd.cut(df['item_price_clean'], bins=price_bins, labels=price_labels)

    print(f"\n【价格区间分布】")
    price_dist = df['price_range'].value_counts().sort_index()
    for price_range, count in price_dist.items():
        percentage = (count / len(df)) * 100
        print(f"  {price_range:10s}: {count:4d} 件 ({percentage:5.2f}%)")

    print()
    return price_dist


def quantity_analysis(df):
    """数量分析"""
    print("=" * 80)
    print("7. 数量分析")
    print("=" * 80)

    print(f"【订购数量统计】")
    print(f"  总订购数量: {df['quantity'].sum():,}")
    print(f"  平均订购数量: {df['quantity'].mean():.2f}")
    print(f"  最大单次订购: {df['quantity'].max()}")

    quantity_dist = df['quantity'].value_counts().sort_index()
    print(f"\n【数量分布】")
    for qty, count in quantity_dist.items():
        percentage = (count / len(df)) * 100
        print(f"  数量 {qty}: {count:4d} 次 ({percentage:5.2f}%)")

    print()


def choice_analysis(df):
    """选项分析"""
    print("=" * 80)
    print("8. 选项分析")
    print("=" * 80)

    # 有选项的商品比例
    has_choice = df['choice_description'].notna().sum()
    no_choice = df['choice_description'].isna().sum()
    total = len(df)

    print(f"【选项统计】")
    print(f"  有选项的商品: {has_choice:4d} ({has_choice/total*100:.2f}%)")
    print(f"  无选项的商品: {no_choice:4d} ({no_choice/total*100:.2f}%)")

    print()


def load_rfm_data(file_path='data/rfm.csv', customer_col='customer_id', 
                  date_col='order_date', amount_col='amount'):
    """加载RFM数据"""
    print("=" * 80)
    print("RFM 数据加载")
    print("=" * 80)
    
    try:
        # 尝试读取RFM数据文件
        df = pd.read_csv(file_path)
        if df.empty:
            raise ValueError("RFM数据文件为空")
        print(f"✓ 成功加载RFM数据文件: {file_path}")
        print(f"✓ 数据形状: {df.shape[0]} 行 × {df.shape[1]} 列")
        
        # 检查必需的列是否存在
        required_cols = [customer_col, date_col, amount_col]
        missing_cols = [col for col in required_cols if col not in df.columns]
        if missing_cols:
            raise ValueError(f"缺少必需的列: {missing_cols}")
        
        # 转换日期列
        df[date_col] = pd.to_datetime(df[date_col])
        print(f"✓ 日期列已转换为datetime类型")
        
        return df, customer_col, date_col, amount_col
    except (FileNotFoundError, ValueError, pd.errors.EmptyDataError) as e:
        print(f"⚠ RFM数据文件不存在或为空，将使用Chipotle数据进行简化RFM分析")
        return None, None, None, None


def calculate_rfm_from_chipotle(df):
    """从Chipotle数据计算RFM指标（使用order_id作为客户ID）"""
    print("=" * 80)
    print("RFM 分析 - 基于Chipotle数据")
    print("=" * 80)
    
    # 使用order_id作为客户ID
    # 由于没有真实的时间数据，我们使用order_id的顺序来模拟时间
    # 假设order_id越大，订单越新
    
    # 计算每个订单的总金额和商品数量
    order_summary = df.groupby('order_id').agg({
        'total_price': 'sum',
        'item_name': 'count'
    }).rename(columns={'total_price': 'monetary', 'item_name': 'frequency'})
    
    # 计算Recency（假设order_id越大越新）
    max_order_id = df['order_id'].max()
    order_summary['recency'] = max_order_id - order_summary.index
    
    # 重命名索引
    order_summary.index.name = 'customer_id'
    order_summary = order_summary.reset_index()
    
    print(f"✓ 计算了 {len(order_summary)} 个客户的RFM指标")
    print(f"  - Recency范围: {order_summary['recency'].min()} - {order_summary['recency'].max()}")
    print(f"  - Frequency范围: {order_summary['frequency'].min()} - {order_summary['frequency'].max()}")
    print(f"  - Monetary范围: ${order_summary['monetary'].min():.2f} - ${order_summary['monetary'].max():.2f}\n")
    
    return order_summary


def calculate_rfm(df, customer_col, date_col, amount_col, reference_date=None):
    """计算RFM指标"""
    print("=" * 80)
    print("RFM 指标计算")
    print("=" * 80)
    
    # 设置参考日期（默认为数据中的最新日期）
    if reference_date is None:
        reference_date = df[date_col].max()
    else:
        reference_date = pd.to_datetime(reference_date)
    
    print(f"✓ 参考日期: {reference_date.strftime('%Y-%m-%d')}")
    
    # 计算每个客户的RFM指标
    rfm = df.groupby(customer_col).agg({
        date_col: lambda x: (reference_date - x.max()).days,  # Recency: 最近一次购买距离现在的天数
        customer_col: 'count',  # Frequency: 购买次数
        amount_col: 'sum'  # Monetary: 总消费金额
    }).rename(columns={
        date_col: 'recency',
        customer_col: 'frequency',
        amount_col: 'monetary'
    })
    
    rfm.index.name = 'customer_id'
    rfm = rfm.reset_index()
    
    print(f"✓ 计算了 {len(rfm)} 个客户的RFM指标")
    print(f"  - Recency范围: {rfm['recency'].min()} - {rfm['recency'].max()} 天")
    print(f"  - Frequency范围: {rfm['frequency'].min()} - {rfm['frequency'].max()} 次")
    print(f"  - Monetary范围: ${rfm['monetary'].min():.2f} - ${rfm['monetary'].max():.2f}\n")
    
    return rfm


def score_rfm(rfm_df, quantiles=None):
    """对RFM指标进行打分（1-5分）"""
    print("=" * 80)
    print("RFM 打分")
    print("=" * 80)
    
    rfm_scored = rfm_df.copy()
    
    # 使用rank方法处理重复值，然后使用cut进行分箱
    # Recency打分：值越小越好（最近购买得分高）
    recency_rank = rfm_scored['recency'].rank(method='first', ascending=True)
    rfm_scored['R_score'] = pd.cut(recency_rank, bins=5, labels=[5, 4, 3, 2, 1], include_lowest=True)
    rfm_scored['R_score'] = rfm_scored['R_score'].astype(int)
    
    # Frequency打分：值越大越好
    frequency_rank = rfm_scored['frequency'].rank(method='first', ascending=True)
    rfm_scored['F_score'] = pd.cut(frequency_rank, bins=5, labels=[1, 2, 3, 4, 5], include_lowest=True)
    rfm_scored['F_score'] = rfm_scored['F_score'].astype(int)
    
    # Monetary打分：值越大越好
    monetary_rank = rfm_scored['monetary'].rank(method='first', ascending=True)
    rfm_scored['M_score'] = pd.cut(monetary_rank, bins=5, labels=[1, 2, 3, 4, 5], include_lowest=True)
    rfm_scored['M_score'] = rfm_scored['M_score'].astype(int)
    
    # 计算RFM总分
    rfm_scored['RFM_score'] = rfm_scored['R_score'] + rfm_scored['F_score'] + rfm_scored['M_score']
    
    print("✓ RFM打分完成")
    print(f"  - R_score范围: {rfm_scored['R_score'].min()} - {rfm_scored['R_score'].max()}")
    print(f"  - F_score范围: {rfm_scored['F_score'].min()} - {rfm_scored['F_score'].max()}")
    print(f"  - M_score范围: {rfm_scored['M_score'].min()} - {rfm_scored['M_score'].max()}")
    print(f"  - RFM总分范围: {rfm_scored['RFM_score'].min()} - {rfm_scored['RFM_score'].max()}\n")
    
    return rfm_scored


def segment_customers(rfm_scored):
    """客户分群"""
    print("=" * 80)
    print("客户分群")
    print("=" * 80)
    
    # 定义客户分群规则
    def assign_segment(row):
        r, f, m = row['R_score'], row['F_score'], row['M_score']
        
        # 重要价值客户（高R、高F、高M）
        if r >= 4 and f >= 4 and m >= 4:
            return '重要价值客户'
        # 重要发展客户（高R、低F、高M）
        elif r >= 4 and f < 3 and m >= 4:
            return '重要发展客户'
        # 重要保持客户（低R、高F、高M）
        elif r < 3 and f >= 4 and m >= 4:
            return '重要保持客户'
        # 重要挽留客户（低R、低F、高M）
        elif r < 3 and f < 3 and m >= 4:
            return '重要挽留客户'
        # 一般价值客户（高R、高F、低M）
        elif r >= 4 and f >= 4 and m < 3:
            return '一般价值客户'
        # 一般发展客户（高R、低F、低M）
        elif r >= 4 and f < 3 and m < 3:
            return '一般发展客户'
        # 一般保持客户（低R、高F、低M）
        elif r < 3 and f >= 4 and m < 3:
            return '一般保持客户'
        # 一般挽留客户（低R、低F、低M）
        else:
            return '一般挽留客户'
    
    rfm_scored['segment'] = rfm_scored.apply(assign_segment, axis=1)
    
    # 统计各分群客户数量
    segment_counts = rfm_scored['segment'].value_counts()
    print("【客户分群统计】")
    for segment, count in segment_counts.items():
        percentage = (count / len(rfm_scored)) * 100
        print(f"  {segment:15s}: {count:5d} 人 ({percentage:5.2f}%)")
    
    # 各分群的平均RFM值
    print("\n【各分群平均RFM值】")
    segment_stats = rfm_scored.groupby('segment').agg({
        'recency': 'mean',
        'frequency': 'mean',
        'monetary': 'mean',
        'R_score': 'mean',
        'F_score': 'mean',
        'M_score': 'mean'
    }).round(2)
    print(segment_stats)
    print()
    
    return rfm_scored


def visualize_rfm(rfm_scored):
    """RFM分析可视化"""
    print("=" * 80)
    print("RFM 可视化")
    print("=" * 80)
    
    fig = plt.figure(figsize=(20, 12))
    
    # 1. RFM得分分布
    ax1 = plt.subplot(2, 3, 1)
    rfm_scored[['R_score', 'F_score', 'M_score']].boxplot(ax=ax1)
    ax1.set_title('RFM得分分布', fontsize=14, fontweight='bold')
    ax1.set_ylabel('得分')
    ax1.set_xticklabels(['R得分', 'F得分', 'M得分'])
    
    # 2. 客户分群分布
    ax2 = plt.subplot(2, 3, 2)
    segment_counts = rfm_scored['segment'].value_counts()
    segment_counts.plot(kind='bar', ax=ax2, color='steelblue', alpha=0.7)
    ax2.set_title('客户分群分布', fontsize=14, fontweight='bold')
    ax2.set_xlabel('客户分群')
    ax2.set_ylabel('客户数量')
    ax2.tick_params(axis='x', rotation=45)
    
    # 3. RFM总分分布
    ax3 = plt.subplot(2, 3, 3)
    ax3.hist(rfm_scored['RFM_score'], bins=15, color='green', alpha=0.7, edgecolor='black')
    ax3.set_title('RFM总分分布', fontsize=14, fontweight='bold')
    ax3.set_xlabel('RFM总分')
    ax3.set_ylabel('客户数量')
    ax3.axvline(rfm_scored['RFM_score'].mean(), color='red', linestyle='--', 
                linewidth=2, label=f'均值: {rfm_scored["RFM_score"].mean():.2f}')
    ax3.legend()
    
    # 4. R vs F 散点图
    ax4 = plt.subplot(2, 3, 4)
    scatter = ax4.scatter(rfm_scored['R_score'], rfm_scored['F_score'], 
                          c=rfm_scored['M_score'], cmap='viridis', 
                          alpha=0.6, s=50)
    ax4.set_title('R得分 vs F得分（颜色表示M得分）', fontsize=14, fontweight='bold')
    ax4.set_xlabel('R得分 (Recency)')
    ax4.set_ylabel('F得分 (Frequency)')
    plt.colorbar(scatter, ax=ax4, label='M得分')
    
    # 5. 各分群的平均消费金额
    ax5 = plt.subplot(2, 3, 5)
    segment_monetary = rfm_scored.groupby('segment')['monetary'].mean().sort_values(ascending=False)
    segment_monetary.plot(kind='barh', ax=ax5, color='coral', alpha=0.7)
    ax5.set_title('各分群平均消费金额', fontsize=14, fontweight='bold')
    ax5.set_xlabel('平均消费金额')
    plt.gca().invert_yaxis()
    
    # 6. 各分群的客户占比（饼图）
    ax6 = plt.subplot(2, 3, 6)
    segment_counts = rfm_scored['segment'].value_counts()
    ax6.pie(segment_counts.values, labels=segment_counts.index, autopct='%1.1f%%',
            startangle=90, textprops={'fontsize': 8})
    ax6.set_title('客户分群占比', fontsize=14, fontweight='bold')
    
    plt.tight_layout()
    plt.savefig('rfm_analysis.png', dpi=300, bbox_inches='tight')
    print("✓ RFM可视化图表已保存为 'rfm_analysis.png'")
    plt.show()
    print()


def rfm_analysis(rfm_df=None, chipotle_df=None):
    """RFM分析主函数"""
    print("\n" + "=" * 80)
    print(" " * 25 + "RFM 客户价值分析")
    print("=" * 80 + "\n")
    
    # 如果提供了RFM数据，使用RFM数据
    if rfm_df is not None:
        rfm = calculate_rfm(rfm_df, 'customer_id', 'order_date', 'amount')
    # 否则使用Chipotle数据
    elif chipotle_df is not None:
        rfm = calculate_rfm_from_chipotle(chipotle_df)
    else:
        print("⚠ 未提供数据，无法进行RFM分析")
        return None
    
    # 对RFM进行打分
    rfm_scored = score_rfm(rfm)
    
    # 客户分群
    rfm_scored = segment_customers(rfm_scored)
    
    # 可视化
    visualize_rfm(rfm_scored)
    
    print("=" * 80)
    print(" " * 25 + "RFM分析完成！")
    print("=" * 80 + "\n")
    
    return rfm_scored


def visualize_data(df, item_counts, item_revenue, order_totals, items_per_order, price_dist):
    """数据可视化"""
    print("=" * 80)
    print("9. 数据可视化")
    print("=" * 80)

    fig = plt.figure(figsize=(20, 12))

    # 1. 最受欢迎的商品（前15）
    ax1 = plt.subplot(2, 3, 1)
    top_items = item_counts.head(15)
    top_items.plot(kind='barh', ax=ax1, color='steelblue')
    ax1.set_title('Top 15 Most Popular Items', fontsize=14, fontweight='bold')
    ax1.set_xlabel('Order Count')
    ax1.set_ylabel('Item Name')
    plt.gca().invert_yaxis()

    # 2. 销售额最高的商品（前15）
    ax2 = plt.subplot(2, 3, 2)
    top_revenue = item_revenue.head(15)
    top_revenue.plot(kind='barh', ax=ax2, color='coral')
    ax2.set_title('Top 15 Items by Revenue', fontsize=14, fontweight='bold')
    ax2.set_xlabel('Revenue ($)')
    ax2.set_ylabel('Item Name')
    plt.gca().invert_yaxis()

    # 3. 订单金额分布
    ax3 = plt.subplot(2, 3, 3)
    ax3.hist(order_totals, bins=50, color='green', alpha=0.7, edgecolor='black')
    ax3.set_title('Order Total Distribution', fontsize=14, fontweight='bold')
    ax3.set_xlabel('Order Total ($)')
    ax3.set_ylabel('Frequency')
    ax3.axvline(order_totals.mean(), color='red', linestyle='--', linewidth=2, label=f'Mean: ${order_totals.mean():.2f}')
    ax3.legend()

    # 4. 每单商品数分布
    ax4 = plt.subplot(2, 3, 4)
    items_per_order.value_counts().sort_index().plot(kind='bar', ax=ax4, color='purple', alpha=0.7)
    ax4.set_title('Items per Order Distribution', fontsize=14, fontweight='bold')
    ax4.set_xlabel('Number of Items')
    ax4.set_ylabel('Number of Orders')
    ax4.tick_params(axis='x', rotation=0)

    # 5. 价格区间分布
    ax5 = plt.subplot(2, 3, 5)
    price_dist.plot(kind='bar', ax=ax5, color='orange', alpha=0.7)
    ax5.set_title('Price Range Distribution', fontsize=14, fontweight='bold')
    ax5.set_xlabel('Price Range')
    ax5.set_ylabel('Count')
    ax5.tick_params(axis='x', rotation=45)

    # 6. 商品价格分布
    ax6 = plt.subplot(2, 3, 6)
    ax6.hist(df['item_price_clean'], bins=30, color='teal', alpha=0.7, edgecolor='black')
    ax6.set_title('Item Price Distribution', fontsize=14, fontweight='bold')
    ax6.set_xlabel('Price ($)')
    ax6.set_ylabel('Frequency')
    ax6.axvline(df['item_price_clean'].mean(), color='red', linestyle='--', linewidth=2,
                label=f'Mean: ${df["item_price_clean"].mean():.2f}')
    ax6.legend()

    plt.tight_layout()
    plt.savefig('chipotle_analysis.png', dpi=300, bbox_inches='tight')
    print("✓ 可视化图表已保存为 'chipotle_analysis.png'")
    plt.show()
    print()


def main():
    """主函数"""
    print("\n" + "=" * 80)
    print(" " * 20 + "CHIPOTLE 数据分析报告")
    print("=" * 80 + "\n")

    # 1. 加载数据
    df = load_data()

    # 2. 基本信息
    basic_info(df)

    # 3. 数据清洗
    df = clean_data(df)

    # 4. 订单分析
    items_per_order, order_totals = order_analysis(df)

    # 5. 商品分析
    item_counts, item_quantity, item_revenue, item_avg_price = item_analysis(df)

    # 6. 价格分析
    price_dist = price_analysis(df)

    # 7. 数量分析
    quantity_analysis(df)

    # 8. 选项分析
    choice_analysis(df)

    # 9. 数据可视化
    visualize_data(df, item_counts, item_revenue, order_totals, items_per_order, price_dist)

    # 10. RFM分析
    rfm_result = rfm_analysis(chipotle_df=df)

    print("=" * 80)
    print(" " * 25 + "分析完成！")
    print("=" * 80 + "\n")


if __name__ == '__main__':
    main()
